import requests
from lxml import etree
import re
import threading
import queue

glock=threading.Lock()

kv={
    'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
}

class productor(threading.Thread):

    kv={
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'
    }

    def __init__(self,getUrl,imgUrl,*args,**kwargs):
        self.getUrl=getUrl
        self.imgUrl=imgUrl
        super(productor,self).__init__(*args,**kwargs)

    def run(self):
        while True:
            if self.getUrl.empty():
                break
            url=self.getUrl.get()
            self.getInfo(url)

    def getInfo(self,url):
        try:
            r=requests.get(url,headers=self.kv)
            r.encoding=r.apparent_encoding
            html=etree.HTML(r.text)
            div=html.xpath('//div[@class="random_picture"]')[0]
            imgs=div.xpath('.//img[contains(@class,"img-responsive")]')
            for img in imgs:
                imgUrl=img.get('data-backup')
                alt=img.get('alt')
                jpg=imgUrl.split('.')[-1]
                alt=re.sub(r'[\.。！，!?？, ]','',alt)
                filename=alt+'.'+jpg
                path='images/imgs/'+filename
                self.imgUrl.put((imgUrl,path))
        except:
            print('%s这个链接下载出错'%url)




class comsumer(threading.Thread):
    def __init__(self,getUrl,imgUrl,*args,**kwargs):
        self.getUrl=getUrl
        self.imgUrl=imgUrl
        super(comsumer,self).__init__(*args,**kwargs)

    def run(self):
        while True:
            if self.imgUrl.empty() and self.getUrl.empty():
                break
            imgUrl,path=self.imgUrl.get()
            r=requests.get(imgUrl)
            img_=r.content
            print('下载了%s'%path)
            with open(path,'wb') as f:
                f.write(img_)
                f.close()


def main(getUrl,imgUrl):
    for x in range(50,60):
        url='http://www.doutula.com/photo/list/?page=%d'%x
        getUrl.put(url)
    for t1 in range(5):
        t=productor(getUrl,imgUrl)
        t.start()
    for t2 in range(5):
        t=comsumer(getUrl,imgUrl)
        t.start()


if __name__=='__main__':
    getUrl=queue.Queue(10)
    imgUrl=queue.Queue(500)
    main(getUrl,imgUrl)
